#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
数据映射和处理模块
负责API数据到数据库字段的映射、转换和处理
支持深层字段访问、JSON解析和数组展开
"""

import logging
import yaml
import json
import itertools
from typing import Dict, Any, List, Optional, Union, Tuple
from datetime import datetime
import re
from html import unescape


class FieldPathResolver:
    """
    字段路径解析器
    支持深层字段访问、JSON解析和数组展开
    """
    
    @staticmethod
    def get_value_by_path(data: Dict[str, Any], path: str) -> Any:
        """
        根据路径获取数据值
        
        支持的路径格式：
        - simple_field: 简单字段
        - detail.field_name: 嵌套字段
        - labels[0].track_tags: JSON数组索引访问
        - detail.array[].field: 数组展开（返回所有元素的值列表）
        
        Args:
            data: 数据字典
            path: 字段路径
            
        Returns:
            字段值或值列表（数组展开时）
        """
        if not path or not data:
            return None
        
        try:
            # 检查是否是数组展开路径
            if '[]' in path:
                return FieldPathResolver._get_array_values(data, path)
            
            # 处理普通路径
            return FieldPathResolver._get_single_value(data, path)
            
        except Exception as e:
            logging.getLogger(__name__).warning(f"字段路径解析失败: {path}, 错误: {e}")
            return None
    
    @staticmethod
    def _get_single_value(data: Dict[str, Any], path: str) -> Any:
        """
        获取单个值
        
        Args:
            data: 数据字典
            path: 字段路径
            
        Returns:
            字段值
        """
        parts = FieldPathResolver._split_path(path)
        current_data = data
        
        for part in parts:
            if current_data is None:
                return None
            
            # 处理数组索引 [0]
            if '[' in part and ']' in part and '[]' not in part:
                field_name, index_part = part.split('[', 1)
                index = int(index_part.replace(']', ''))
                
                if field_name:
                    # 先获取字段值
                    field_value = current_data.get(field_name) if isinstance(current_data, dict) else None
                    
                    # 如果是JSON字符串，先解析
                    if isinstance(field_value, str):
                        try:
                            field_value = json.loads(field_value)
                        except (json.JSONDecodeError, TypeError):
                            return None
                    
                    current_data = field_value
                else:
                    # 直接索引当前数据
                    pass
                
                if isinstance(current_data, (list, tuple)) and len(current_data) > index:
                    current_data = current_data[index]
                else:
                    return None
            else:
                # 普通字段访问
                if isinstance(current_data, dict):
                    current_data = current_data.get(part)
                    
                    # 如果获取到的值是JSON字符串，尝试解析
                    # 这对于那些可能存储为JSON字符串的数组字段很重要
                    if isinstance(current_data, str) and (current_data.startswith('[') or current_data.startswith('{')):
                        try:
                            parsed_data = json.loads(current_data)
                            current_data = parsed_data
                        except (json.JSONDecodeError, TypeError):
                            # 解析失败时保持原值，可能不是JSON格式
                            pass
                else:
                    return None
        
        return current_data
    
    @staticmethod
    def _get_array_values(data: Dict[str, Any], path: str) -> List[Any]:
        """
        获取数组展开的所有值
        
        Args:
            data: 数据字典
            path: 数组路径（包含[]）
            
        Returns:
            值列表
        """
        # 分割数组路径：detail.top_shareholders_company[].shareholderName
        # 分为：detail.top_shareholders_company 和 shareholderName
        array_part, field_part = path.split('[]', 1)
        field_part = field_part.lstrip('.')  # 移除开头的点
        
        # 获取数组数据
        array_data = FieldPathResolver._get_single_value(data, array_part)
        
        # 如果是JSON字符串，先解析为对象
        if isinstance(array_data, str):
            try:
                array_data = json.loads(array_data)
            except (json.JSONDecodeError, TypeError):
                logging.getLogger(__name__).warning(f"无法解析JSON字符串: {array_data}")
                return []
        
        if not isinstance(array_data, list):
            return []
        
        # 从每个数组元素中提取字段值
        values = []
        for item in array_data:
            if isinstance(item, dict):
                if field_part:
                    # 提取子字段
                    value = FieldPathResolver._get_single_value(item, field_part)
                else:
                    # 直接使用整个元素
                    value = item
                values.append(value)
        
        return values
    
    @staticmethod
    def _split_path(path: str) -> List[str]:
        """
        分割字段路径
        
        Args:
            path: 字段路径
            
        Returns:
            路径部分列表
        """
        parts = []
        current_part = ""
        bracket_count = 0
        
        for char in path:
            if char == '.':
                if bracket_count == 0:
                    if current_part:
                        parts.append(current_part)
                        current_part = ""
                else:
                    current_part += char
            elif char == '[':
                bracket_count += 1
                current_part += char
            elif char == ']':
                bracket_count -= 1
                current_part += char
            else:
                current_part += char
        
        if current_part:
            parts.append(current_part)
        
        return parts


class DataProcessor:
    """
    数据处理器
    负责API数据的映射、转换和处理
    """
    
    def __init__(self, config_path: str = 'config.yaml'):
        """
        初始化数据处理器
        
        Args:
            config_path: 配置文件路径
        """
        self.logger = logging.getLogger(__name__)
        self.config = self._load_config(config_path)
        self.field_mapping = self.config.get('field_mapping', {})
        self.date_format = self.config['sync'].get('date_format', '%Y-%m-%d %H:%M:%S')
        self.path_resolver = FieldPathResolver()
    
    def _load_config(self, config_path: str) -> Dict[str, Any]:
        """
        加载配置文件
        
        Args:
            config_path: 配置文件路径
            
        Returns:
            配置字典
        """
        try:
            with open(config_path, 'r', encoding='utf-8') as f:
                config = yaml.safe_load(f)
            return config
        except Exception as e:
            self.logger.error(f"加载配置文件失败: {e}")
            raise
    
    def _safe_get_value(self, data: Dict[str, Any], key: str, default: Any = None) -> Any:
        """
        安全获取字典值
        
        Args:
            data: 数据字典
            key: 键名
            default: 默认值
            
        Returns:
            值
        """
        value = data.get(key, default)
        
        # 处理空值情况
        if value in ['', [], None, 'null', 'NULL']:
            return None
        
        return value
    
    def _convert_datetime(self, datetime_str: Optional[str]) -> Optional[str]:
        """
        转换日期时间格式
        
        Args:
            datetime_str: 日期时间字符串
            
        Returns:
            格式化后的日期时间字符串
        """
        if not datetime_str or datetime_str in ['', 'null', 'NULL']:
            return None
        
        try:
            # 尝试解析不同的日期格式
            formats = [
                '%Y-%m-%d %H:%M:%S',
                '%Y-%m-%d',
                '%Y/%m/%d %H:%M:%S',
                '%Y/%m/%d'
            ]
            
            for fmt in formats:
                try:
                    dt = datetime.strptime(datetime_str, fmt)
                    return dt.strftime(self.date_format)
                except ValueError:
                    continue
            
            # 如果都不匹配，返回原值
            self.logger.warning(f"无法解析日期格式: {datetime_str}")
            return datetime_str
            
        except Exception as e:
            self.logger.warning(f"日期转换失败: {datetime_str}, 错误: {e}")
            return datetime_str
    
    def _clean_html(self, html_str: Optional[str]) -> Optional[str]:
        """
        清理HTML标签
        
        Args:
            html_str: HTML字符串
            
        Returns:
            清理后的文本
        """
        if not html_str or html_str in ['', 'null', 'NULL']:
            return None
        
        try:
            # 移除HTML标签
            clean_text = re.sub(r'<[^>]+>', '', str(html_str))
            # 解码HTML实体
            clean_text = unescape(clean_text)
            # 移除多余的空白字符
            clean_text = re.sub(r'\s+', ' ', clean_text).strip()
            
            return clean_text if clean_text else None
            
        except Exception as e:
            self.logger.warning(f"HTML清理失败: {e}")
            return str(html_str)
    
    def _group_array_fields(self, field_mapping: Dict[str, str]) -> Tuple[Dict[str, str], Dict[str, List[str]]]:
        """
        分组普通字段和数组字段
        
        Args:
            field_mapping: 字段映射配置
            
        Returns:
            (普通字段映射, 数组字段分组)
        """
        normal_fields = {}
        array_groups = {}
        
        for db_field, api_path in field_mapping.items():
            if '[]' in api_path:
                # 数组字段
                array_path = api_path.split('[]')[0]  # 获取数组路径部分
                if array_path not in array_groups:
                    array_groups[array_path] = []
                array_groups[array_path].append((db_field, api_path))
            else:
                # 普通字段
                normal_fields[db_field] = api_path
        
        return normal_fields, array_groups
    
    def _process_normal_fields(self, api_data: Dict[str, Any], 
                             field_mapping: Dict[str, str]) -> Dict[str, Any]:
        """
        处理普通字段（非数组字段）
        
        Args:
            api_data: API数据
            field_mapping: 字段映射
            
        Returns:
            处理后的字段字典
        """
        result = {}
        
        for db_field, api_path in field_mapping.items():
            value = self.path_resolver.get_value_by_path(api_data, api_path)
            
            # 特殊字段处理
            if db_field in ['dic_start', 'dic_end', 'created_at', 'updated_at', 'register_date', 'ratify_date']:
                value = self._convert_datetime(value)
            elif db_field in ['BDJBXX', 'JYLC', 'content']:
                value = self._clean_html(value)
            elif isinstance(value, (list, dict)):
                # 如果是复杂类型，转换为JSON字符串
                try:
                    value = json.dumps(value, ensure_ascii=False) if value else None
                except Exception:
                    value = str(value) if value else None
            
            result[db_field] = value
        
        return result
    
    def _expand_array_fields(self, api_data: Dict[str, Any], 
                           array_groups: Dict[str, List[Tuple[str, str]]]) -> List[Dict[str, Any]]:
        """
        展开数组字段，生成笛卡尔积记录
        
        Args:
            api_data: API数据
            array_groups: 数组字段分组
            
        Returns:
            展开后的记录列表
        """
        if not array_groups:
            return [{}]
        
        # 收集所有数组数据
        arrays_data = {}
        
        for array_path, field_list in array_groups.items():
            # 获取数组数据
            array_data = self.path_resolver.get_value_by_path(api_data, array_path)
            
            if isinstance(array_data, list) and len(array_data) > 0:
                arrays_data[array_path] = {
                    'data': array_data,
                    'fields': field_list
                }
        
        # 如果没有有效的数组数据，返回空记录
        if not arrays_data:
            # 创建包含所有数组字段的空记录
            empty_record = {}
            for field_list in array_groups.values():
                for db_field, _ in field_list:
                    empty_record[db_field] = None
            return [empty_record]
        
        # 生成笛卡尔积
        array_names = list(arrays_data.keys())
        array_values = [arrays_data[name]['data'] for name in array_names]
        
        expanded_records = []
        for combination in itertools.product(*array_values):
            record = {}
            
            # 为每个数组字段添加对应的值
            for i, array_name in enumerate(array_names):
                array_item = combination[i]
                field_list = arrays_data[array_name]['fields']
                
                for db_field, api_path in field_list:
                    # 从数组元素中提取字段值
                    value = self.path_resolver.get_value_by_path(api_data, api_path)
                    
                    # 如果是数组路径，从当前组合项中获取值
                    if '[]' in api_path and isinstance(array_item, dict):
                        field_part = api_path.split('[]', 1)[1].lstrip('.')
                        if field_part:
                            value = self.path_resolver.get_value_by_path(array_item, field_part)
                        else:
                            value = array_item
                    
                    # 特殊字段处理
                    if db_field in ['ratify_date', 'register_date']:
                        value = self._convert_datetime(value)
                    elif db_field in ['content']:
                        value = self._clean_html(value)
                    
                    record[db_field] = str(value) if value is not None else None
            
            # 确保所有数组字段都有值（即使是None）
            for field_list in array_groups.values():
                for db_field, _ in field_list:
                    if db_field not in record:
                        record[db_field] = None
            
            expanded_records.append(record)
        
        return expanded_records
    
    def validate_record(self, record: Dict[str, Any], table_name: str) -> bool:
        """
        验证记录数据
        
        Args:
            record: 记录数据
            table_name: 表名
            
        Returns:
            是否有效
        """
        try:
            # 基础验证：检查记录是否为有效的字典
            if not record or not isinstance(record, dict):
                return False
            
            return True
            
        except Exception as e:
            self.logger.error(f"记录验证失败: {e}")
            return False
    
    def get_table_name_by_biz_sys(self, biz_sys: str) -> Optional[str]:
        """
        根据业务系统获取表名
        
        Args:
            biz_sys: 业务系统名称
            
        Returns:
            表名
        """
        biz_sys_mapping = self.config.get('biz_sys_mapping', {})
        table_name = biz_sys_mapping.get(biz_sys)
        return table_name
    
    def process_data_by_biz_sys(self, api_data_list: List[Dict[str, Any]], 
                               biz_sys: str) -> List[Dict[str, Any]]:
        """
        根据业务系统处理数据（通用方法）
        
        Args:
            api_data_list: API数据列表
            biz_sys: 业务系统名称
            
        Returns:
            处理后的数据列表
        """
        table_name = self.get_table_name_by_biz_sys(biz_sys)
        
        if not table_name:
            self.logger.warning(f"未找到业务系统 '{biz_sys}' 对应的表名")
            return []
        
        # 使用通用的数据处理方法
        processed_data = self.process_table_data(api_data_list, table_name)
        
        # 验证处理后的数据
        valid_records = []
        for record in processed_data:
            if self.validate_record(record, table_name):
                valid_records.append(record)
        
        self.logger.info(f"数据验证完成，有效记录: {len(valid_records)}/{len(processed_data)}")
        return valid_records
    
    def _normalize_field_paths(self, field_mapping: Dict[str, str]) -> Dict[str, str]:
        """
        标准化字段路径，移除 data.list[] 前缀
        
        Args:
            field_mapping: 原始字段映射
            
        Returns:
            标准化后的字段映射
        """
        normalized_mapping = {}
        
        for db_field, api_path in field_mapping.items():
            # 移除 data.list[] 前缀，因为我们处理的是单个记录
            if api_path.startswith('data.list[].'):
                normalized_path = api_path[12:]  # 移除 'data.list[].' 前缀
            elif api_path.startswith('data.list[]'):
                normalized_path = api_path[11:]  # 移除 'data.list[]' 前缀
            else:
                normalized_path = api_path
            
            normalized_mapping[db_field] = normalized_path
        
        return normalized_mapping

    def process_table_data(self, api_data_list: List[Dict[str, Any]], 
                          table_name: str) -> List[Dict[str, Any]]:
        """
        通用的表数据处理方法
        
        Args:
            api_data_list: API数据列表
            table_name: 表名
            
        Returns:
            处理后的数据库记录列表（展开后可能比输入多）
        """
        if not api_data_list:
            return []
        
        try:
            # 从配置中获取表的字段映射配置
            table_config = self.field_mapping.get(table_name, {})
            field_mapping = table_config.get('fields', {})
            
            if not field_mapping:
                self.logger.warning(f"表 '{table_name}' 没有配置字段映射")
                return []
            
            # 标准化字段路径（移除 data.list[] 前缀）
            normalized_mapping = self._normalize_field_paths(field_mapping)
            
            # 分组普通字段和数组字段
            normal_fields, array_groups = self._group_array_fields(normalized_mapping)
            
            processed_records = []
            
            for api_data in api_data_list:
                try:
                    # 处理普通字段
                    base_record = self._process_normal_fields(api_data, normal_fields)
                    
                    # 展开数组字段，生成多条记录
                    expanded_records = self._expand_array_fields(api_data, array_groups)
                    
                    # 为每个展开的记录添加基础字段
                    for expanded_record in expanded_records:
                        # 合并基础记录和展开记录
                        final_record = base_record.copy()
                        final_record.update(expanded_record)
                        
                        # 设置默认值
                        if 'created_at' not in final_record or not final_record['created_at']:
                            final_record['created_at'] = datetime.now().strftime(self.date_format)
                        if 'updated_at' not in final_record or not final_record['updated_at']:
                            final_record['updated_at'] = datetime.now().strftime(self.date_format)
                        
                        processed_records.append(final_record)
                    
                except Exception as e:
                    self.logger.error(f"处理单条记录失败: {e}, 数据: {api_data.get('id', 'unknown')}")
                    continue
            
            self.logger.info(f"{table_name}数据处理完成，输入: {len(api_data_list)}, 输出: {len(processed_records)}")
            return processed_records
            
        except Exception as e:
            self.logger.error(f"{table_name}批量数据处理失败: {e}")
            raise


class DataTransformer:
    """
    数据转换器
    提供通用的数据转换功能
    """
    
    @staticmethod
    def safe_int(value: Any, default: int = 0) -> int:
        """
        安全转换为整数
        
        Args:
            value: 值
            default: 默认值
            
        Returns:
            整数值
        """
        try:
            if value in ['', None, 'null', 'NULL']:
                return default
            return int(float(str(value)))
        except (ValueError, TypeError):
            return default
    
    @staticmethod
    def safe_float(value: Any, default: float = 0.0) -> float:
        """
        安全转换为浮点数
        
        Args:
            value: 值
            default: 默认值
            
        Returns:
            浮点数值
        """
        try:
            if value in ['', None, 'null', 'NULL']:
                return default
            return float(str(value))
        except (ValueError, TypeError):
            return default
    
    @staticmethod
    def safe_str(value: Any, default: str = '') -> str:
        """
        安全转换为字符串
        
        Args:
            value: 值
            default: 默认值
            
        Returns:
            字符串值
        """
        try:
            if value in [None, 'null', 'NULL']:
                return default
            return str(value).strip()
        except (ValueError, TypeError):
            return default
    
    def process_operational_data(self, api_data_list: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
        """
        处理房源运营信息数据（专门用于bjdc_lease_house表）
        
        Args:
            api_data_list: 房源运营信息API数据列表
            
        Returns:
            处理后的数据库记录列表
        """
        if not api_data_list:
            return []
        
        try:
            # 获取bjdc_lease_house表的字段映射配置
            table_config = self.field_mapping.get('bjdc_lease_house', {})
            field_mapping = table_config.get('fields', {})
            
            if not field_mapping:
                self.logger.warning("bjdc_lease_house表没有配置字段映射")
                return []
            
            processed_records = []
            
            for api_data in api_data_list:
                try:
                    # 获取项目基础信息
                    project_id = api_data.get('projectId')
                    target_list = api_data.get('targetList', [])
                    
                    if not target_list:
                        # 如果没有targetList，创建一条只有projectId的记录
                        base_record = {
                            'projectId': project_id,
                            'created_at': datetime.now().strftime(self.date_format),
                            'updated_at': datetime.now().strftime(self.date_format)
                        }
                        
                        # 设置其他字段为None
                        for db_field in field_mapping.keys():
                            if db_field not in base_record:
                                if db_field == 'is_delete':
                                    base_record[db_field] = '0'
                                else:
                                    base_record[db_field] = None
                        
                        processed_records.append(base_record)
                        continue
                    
                    # 遍历每个target，为每个target创建记录
                    for target in target_list:
                        try:
                            record = {
                                'projectId': project_id,
                                'created_at': datetime.now().strftime(self.date_format),
                                'updated_at': datetime.now().strftime(self.date_format)
                            }
                            
                            # 处理每个字段映射
                            for db_field, api_path in field_mapping.items():
                                if db_field in ['created_at', 'updated_at', 'projectId']:
                                    continue  # 这些字段已经处理了
                                
                                if api_path == '0':  # 默认值
                                    record[db_field] = '0'
                                    continue
                                
                                value = None
                                
                                if api_path.startswith('targetList[].'):
                                    # 从target中获取值
                                    field_path = api_path[13:]  # 移除 'targetList[].' 前缀
                                    value = self._get_target_value(target, field_path)
                                else:
                                    # 从项目根级别获取值（如projectId）
                                    value = self.path_resolver.get_value_by_path(api_data, api_path)
                                
                                # 处理特殊字段类型
                                if db_field == 'allFloor' and value is not None:
                                    try:
                                        value = str(int(value)) if value != '' else None
                                    except (ValueError, TypeError):
                                        value = None
                                
                                record[db_field] = value
                            
                            processed_records.append(record)
                            
                        except Exception as e:
                            self.logger.error(f"处理target失败: {e}, target: {target}")
                            continue
                    
                except Exception as e:
                    self.logger.error(f"处理项目数据失败: {e}, projectId: {api_data.get('projectId', 'unknown')}")
                    continue
            
            self.logger.info(f"房源运营信息数据处理完成，输入: {len(api_data_list)}, 输出: {len(processed_records)}")
            return processed_records
            
        except Exception as e:
            self.logger.error(f"房源运营信息批量数据处理失败: {e}")
            raise
    
    def _get_target_value(self, target: Dict[str, Any], field_path: str) -> Any:
        """
        从target数据中获取字段值
        
        Args:
            target: target数据字典
            field_path: 字段路径
            
        Returns:
            字段值
        """
        try:
            # 处理数组字段（如houseLabelList[].houseLabelCode）
            if '[]' in field_path:
                array_part, field_part = field_path.split('[]', 1)
                field_part = field_part.lstrip('.')  # 移除开头的点
                
                # 获取数组数据
                array_data = target.get(array_part)
                if not isinstance(array_data, list) or not array_data:
                    return None
                
                # 如果有子字段，提取第一个元素的子字段值
                if field_part:
                    first_item = array_data[0]
                    if isinstance(first_item, dict):
                        return first_item.get(field_part)
                    return None
                else:
                    # 直接返回第一个元素
                    return array_data[0] if array_data else None
            else:
                # 普通字段访问
                return target.get(field_path)
                
        except Exception as e:
            self.logger.warning(f"获取target字段值失败: {field_path}, 错误: {e}")
            return None